{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow  as tf\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0-beta0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_image, train_labels), (test_image, test_labels) = tf.keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_image = tf.expand_dims(train_image, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_image = tf.expand_dims(test_image, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([60000, 28, 28, 1])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_image = tf.cast(train_image/255, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_image = tf.cast(test_image/255, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = tf.cast(train_labels, tf.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_labels = tf.cast(test_labels, tf.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((train_image, train_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_image, test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<TensorSliceDataset shapes: ((28, 28, 1), ()), types: (tf.float32, tf.int64)>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.repeat(1).shuffle(60000).batch(128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = test_dataset.repeat(1).batch(128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<BatchDataset shapes: ((None, 28, 28, 1), (None,)), types: (tf.float32, tf.int64)>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Conv2D(16, [3,3], activation='relu', input_shape=(None, None, 1)),\n",
    "    tf.keras.layers.Conv2D(32, [3,3], activation='relu'),\n",
    "    tf.keras.layers.GlobalMaxPooling2D(),\n",
    "    tf.keras.layers.Dense(10, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir=os.path.join(\"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.create_file_writer(log_dir + \"/metrics\")\n",
    "file_writer.set_as_default()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"\n",
    "    Returns a custom learning rate that decreases as epochs progress.\n",
    "    \"\"\"\n",
    "    learning_rate = 0.2\n",
    "    if epoch > 5:\n",
    "        learning_rate = 0.02\n",
    "    if epoch > 10:\n",
    "        learning_rate = 0.01\n",
    "    if epoch > 20:\n",
    "        learning_rate = 0.005\n",
    "\n",
    "    tf.summary.scalar('learning rate', data=learning_rate, step=epoch)\n",
    "    return learning_rate\n",
    "\n",
    "lr_callback = tf.keras.callbacks.LearningRateScheduler(lr_schedule)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0803 10:50:27.427461 10288 deprecation.py:323] From c:\\users\\guanghua\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "468/468 [==============================] - 21s 45ms/step - loss: 1.4020 - accuracy: 0.5771 - val_loss: 0.7032 - val_accuracy: 0.7956\n",
      "Epoch 2/5\n",
      "468/468 [==============================] - 25s 54ms/step - loss: 0.5852 - accuracy: 0.8183 - val_loss: 0.4751 - val_accuracy: 0.8522\n",
      "Epoch 3/5\n",
      "468/468 [==============================] - 25s 54ms/step - loss: 0.4335 - accuracy: 0.8657 - val_loss: 0.3766 - val_accuracy: 0.8858\n",
      "Epoch 4/5\n",
      "468/468 [==============================] - 27s 57ms/step - loss: 0.3696 - accuracy: 0.8845 - val_loss: 0.3268 - val_accuracy: 0.8983\n",
      "Epoch 5/5\n",
      "468/468 [==============================] - 28s 59ms/step - loss: 0.3302 - accuracy: 0.8951 - val_loss: 0.2970 - val_accuracy: 0.9086\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2b6b4981978>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(dataset,\n",
    "          epochs=5,\n",
    "          steps_per_epoch=60000//128,\n",
    "          validation_data=test_dataset,\n",
    "          validation_steps=10000//128,\n",
    "          callbacks=[tensorboard_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The tensorboard extension is already loaded. To reload it, use:\n",
      "  %reload_ext tensorboard\n"
     ]
    }
   ],
   "source": [
    "%load_ext tensorboard\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6006 (pid 2980), started 0:09:45 ago. (Use '!kill 2980' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"800\"\n",
       "            src=\"http://localhost:6006\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x2b6d3bb2f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SCALARS 面板主要用于记录诸如准确率、损失和学习率等单个值的变化趋势。在代码中用 tf.summary.scalar() 来将其记录到文件中"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每个图的右下角都有 3 个小图标，第一个是查看大图，第二个是是否对 y 轴对数化，第三个是如果你拖动或者缩放了坐标轴，再重新回到原始位置。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GRAPHS 面板展示出你所构建的网络整体结构，显示数据流的方向和大小，也可以显示训练时每个节点的用时、耗费的内存大小以及参数多少。默认显示的图分为两部分：主图（Main Graph）和辅助节点（Auxiliary Nodes）。其中主图显示的就是网络结构，辅助节点则显示的是初始化、训练、保存等节点。我们可以双击某个节点或者点击节点右上角的 + 来展开查看里面的情况，也可以对齐进行缩放"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DISTRIBUTIONS 主要用来展示网络中各参数随训练步数的增加的变化情况，可以说是 多分位数折线图 的堆叠。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "HISTOGRAMS 和 DISTRIBUTIONS 是对同一数据不同方式的展现。与 DISTRIBUTIONS 不同的是，HISTOGRAMS 可以说是 频数分布直方图 的堆叠。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 记录自定义标量"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "重新调整回归模型并记录自定义学习率。这是如何做：\n",
    "\n",
    "使用创建文件编写器tf.summary.create_file_writer()。\n",
    "\n",
    "定义自定义学习率功能。这将被传递给Keras LearningRateScheduler回调。\n",
    "\n",
    "在学习率功能内，用于tf.summary.scalar()记录自定义学习率。\n",
    "\n",
    "将LearningRateScheduler回调传递给Model.fit（）。\n",
    "\n",
    "通常，要记录自定义标量，您需要使用tf.summary.scalar()文件编写器。文件编写器负责将此运行的数据写入指定的目录，并在使用时隐式使用tf.summary.scalar()。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "468/468 [==============================] - 22s 46ms/step - loss: 14.5074 - accuracy: 0.0992 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 2/30\n",
      "468/468 [==============================] - 26s 55ms/step - loss: 14.5473 - accuracy: 0.0975 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 3/30\n",
      "468/468 [==============================] - 28s 60ms/step - loss: 14.5279 - accuracy: 0.0987 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 4/30\n",
      "468/468 [==============================] - 29s 63ms/step - loss: 14.5484 - accuracy: 0.0974 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 5/30\n",
      "468/468 [==============================] - 31s 67ms/step - loss: 14.5559 - accuracy: 0.0969 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 6/30\n",
      "468/468 [==============================] - 32s 68ms/step - loss: 14.5656 - accuracy: 0.0963 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 7/30\n",
      "468/468 [==============================] - 33s 70ms/step - loss: 14.5110 - accuracy: 0.0997 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 8/30\n",
      "468/468 [==============================] - 30s 65ms/step - loss: 14.5497 - accuracy: 0.0973 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 9/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5860 - accuracy: 0.0951 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 10/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5642 - accuracy: 0.0964 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 11/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5274 - accuracy: 0.0987 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 12/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5529 - accuracy: 0.0971 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 13/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5290 - accuracy: 0.0986 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 14/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5500 - accuracy: 0.0973 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 15/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5640 - accuracy: 0.0964 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 16/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5333 - accuracy: 0.0983 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 17/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5634 - accuracy: 0.0965 - val_loss: 14.5360 - val_accuracy: 0.09825623 \n",
      "Epoch 18/30\n",
      "468/468 [==============================] - 29s 63ms/step - loss: 14.5209 - accuracy: 0.0991 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 19/30\n",
      "468/468 [==============================] - 29s 63ms/step - loss: 14.5998 - accuracy: 0.0942 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 20/30\n",
      "468/468 [==============================] - 30s 63ms/step - loss: 14.5330 - accuracy: 0.0983 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 21/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5400 - accuracy: 0.0979 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 22/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5376 - accuracy: 0.0981 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 23/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5376 - accuracy: 0.0981 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 24/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5613 - accuracy: 0.0966 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 25/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5699 - accuracy: 0.0961 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 26/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5468 - accuracy: 0.0975 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 27/30\n",
      "468/468 [==============================] - 30s 63ms/step - loss: 14.5352 - accuracy: 0.0982 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 28/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5500 - accuracy: 0.0973 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 29/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5492 - accuracy: 0.0973 - val_loss: 14.5360 - val_accuracy: 0.0982\n",
      "Epoch 30/30\n",
      "468/468 [==============================] - 29s 62ms/step - loss: 14.5462 - accuracy: 0.0975 - val_loss: 14.5360 - val_accuracy: 0.0982\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2b6d3bb2cc0>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(dataset,\n",
    "          epochs=30,\n",
    "          steps_per_epoch=60000//128,\n",
    "          validation_data=test_dataset,\n",
    "          validation_steps=10000//128,\n",
    "          callbacks=[tensorboard_callback, lr_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 自定义训练中使用Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_func = tf.keras.losses.SparseCategoricalCrossentropy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(model, x, y):\n",
    "    y_ = model(x)\n",
    "    return loss_func(y, y_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss = tf.keras.metrics.Mean('train_loss')\n",
    "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('train_accuracy')\n",
    "\n",
    "test_loss = tf.keras.metrics.Mean('test_loss')\n",
    "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('test_accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_step(model, images, labels):\n",
    "    with tf.GradientTape() as t:\n",
    "        pred = model(images)\n",
    "        loss_step = loss_func(labels, pred)\n",
    "    grads = t.gradient(loss_step, model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
    "    train_loss(loss_step)\n",
    "    train_accuracy(labels, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_step(model, images, labels):\n",
    "    pred = model(images)\n",
    "    loss_step = loss_func(labels, pred)\n",
    "    test_loss(loss_step)\n",
    "    test_accuracy(labels, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "train_log_dir = 'logs/gradient_tape/' + current_time + '/train'\n",
    "test_log_dir = 'logs/gradient_tape/' + current_time + '/test'\n",
    "train_summary_writer = tf.summary.create_file_writer(train_log_dir)\n",
    "test_summary_writer = tf.summary.create_file_writer(test_log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train():\n",
    "    for epoch in range(10):\n",
    "        for (batch, (images, labels)) in enumerate(dataset):\n",
    "            train_step(model, images, labels)\n",
    "            print('.', end='')\n",
    "    \n",
    "        with train_summary_writer.as_default():\n",
    "            tf.summary.scalar('loss', train_loss.result(), step=epoch)\n",
    "            tf.summary.scalar('accuracy', train_accuracy.result(), step=epoch)\n",
    "        \n",
    "        for (batch, (images, labels)) in enumerate(test_dataset):\n",
    "            test_step(model, images, labels)\n",
    "            print('*', end='')\n",
    "            \n",
    "        with test_summary_writer.as_default():\n",
    "            tf.summary.scalar('loss', test_loss.result(), step=epoch)\n",
    "            tf.summary.scalar('accuracy', test_accuracy.result(), step=epoch)\n",
    "        template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n",
    "        print(template.format(epoch+1,\n",
    "                               train_loss.result(), \n",
    "                               train_accuracy.result()*100,\n",
    "                               test_loss.result(), \n",
    "                               test_accuracy.result()*100))\n",
    "        \n",
    "        train_loss.reset_states()\n",
    "        train_accuracy.reset_states()\n",
    "        test_loss.reset_states()\n",
    "        test_accuracy.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0803 14:22:03.458425  6732 deprecation.py:323] From c:\\users\\guanghua\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1220: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 1, Loss: 1.419425368309021, Accuracy: 58.26667022705078, Test Loss: 0.6785365343093872, Test Accuracy: 80.01000213623047\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 2, Loss: 0.5745379328727722, Accuracy: 82.13166809082031, Test Loss: 0.4586440324783325, Test Accuracy: 85.5999984741211\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 3, Loss: 0.455000102519989, Accuracy: 85.54833221435547, Test Loss: 0.39740514755249023, Test Accuracy: 87.30999755859375\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 4, Loss: 0.4024190902709961, Accuracy: 87.27833557128906, Test Loss: 0.3615242540836334, Test Accuracy: 88.3499984741211\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 5, Loss: 0.3679908812046051, Accuracy: 88.37166595458984, Test Loss: 0.3343818783760071, Test Accuracy: 89.2300033569336\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 6, Loss: 0.3422515094280243, Accuracy: 89.22332763671875, Test Loss: 0.3136812150478363, Test Accuracy: 89.81000518798828\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 7, Loss: 0.3215223252773285, Accuracy: 89.83499908447266, Test Loss: 0.2968979477882385, Test Accuracy: 90.34000396728516\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 8, Loss: 0.30404961109161377, Accuracy: 90.413330078125, Test Loss: 0.2825365960597992, Test Accuracy: 90.8699951171875\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 9, Loss: 0.28920120000839233, Accuracy: 90.90499877929688, Test Loss: 0.27007314562797546, Test Accuracy: 91.43999481201172\n",
      ".....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*******************************************************************************Epoch 10, Loss: 0.2767001986503601, Accuracy: 91.26499938964844, Test Loss: 0.2601126730442047, Test Accuracy: 91.75\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Line magic function `%tensorboard` not found.\n"
     ]
    }
   ],
   "source": [
    "%tensorboard --logdir logs/gradient_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
